import CrawlerConfig;

import time;
import threading;
import os;

'''
    an interface for blog crawler
'''
class BlogCrawler:
    def crawl(self, configPath):
        print 'It\'s a virtual class and no action!';

'''
    update page based blog crawler
'''        
class UpdatePageBlogCrawler(BlogCrawler):
    def __init__(self, config):
        from BlogUpdateCrawler import BlogUpdateCrawler;
        from PageBlogContentCrawler import PageBlogContentCrawler;
        self.updateCrawler = BlogUpdateCrawler(config);
        self.profileCrawler = ProfileCrawler(config);
        self.contentCrawler = PageBlogContentCrawler(config);
        self.config = config;
         
    def crawl(self):
        #1. crawl update page
        threading.Thread(target = self.updateCrawler.crawl).start();
        #2. crawl the posting content
        threadNum = int(self.config.data[self.config.THREAD_NUM_VARNAME]);
        for i in xrange(threadNum):
            time.sleep(5);
            threading.Thread(target = self.contentCrawler.crawl).start();
    
'''
    stream based blog crawler
'''    
class AllinOneStreamBlogCrawler(BlogCrawler):
    def __init__(self, config):
        print 'AllinOneStreamBlogCrawler::__init__(): start!'
        self.streamCrawler = config.streamCrawler;
        self.streamParser = config.streamParser;
        self.streamParserThreadNum = config.streamParserThreadNum;
        self.profileCrawler = config.profileCrawler;
        self.profileCrawlerThreadNum = config.profileCrawlerThreadNum;
        taskId = config.taskId;
        if not os.path.exists(taskId):
            os.makedirs(taskId);

    def crawl(self):
        print 'AllinOneStreamBlogCrawler::crawl(): start!'
        #1. crawl stream
        threading.Thread(target = self.streamCrawler.crawl).start();
        time.sleep(5);
        #2. parse stream content
        for i in range(self.streamParserThreadNum):
            threading.Thread(target = self.streamParser.parse).start();            
        time.sleep(5);
        #3. crawl profile content
        for i in range(self.profileCrawlerThreadNum):
            threading.Thread(target = self.profileCrawler.crawl).start();
        
